/*     */ package com.zimbra.cs.index;
/*     */ 
/*     */ import com.zimbra.common.util.SetUtil;
/*     */ import java.io.File;
/*     */ import java.io.FileWriter;
/*     */ import java.io.IOException;
/*     */ import java.io.PrintStream;
/*     */ import java.io.PrintWriter;
/*     */ import java.util.ArrayList;
/*     */ import java.util.Collection;
/*     */ import java.util.Collections;
/*     */ import java.util.Date;
/*     */ import java.util.HashSet;
/*     */ import java.util.List;
/*     */ import java.util.Set;
/*     */ import org.apache.commons.cli.CommandLine;
/*     */ import org.apache.commons.cli.CommandLineParser;
/*     */ import org.apache.commons.cli.GnuParser;
/*     */ import org.apache.commons.cli.HelpFormatter;
/*     */ import org.apache.commons.cli.Option;
/*     */ import org.apache.commons.cli.Options;
/*     */ import org.apache.lucene.document.DateTools;
/*     */ import org.apache.lucene.document.Document;
/*     */ import org.apache.lucene.document.Fieldable;
/*     */ import org.apache.lucene.index.CheckIndex;
/*     */ import org.apache.lucene.index.CheckIndex.Status;
/*     */ import org.apache.lucene.index.IndexReader;
/*     */ import org.apache.lucene.index.IndexReader.FieldOption;
/*     */ import org.apache.lucene.index.Term;
/*     */ import org.apache.lucene.index.TermEnum;
/*     */ import org.apache.lucene.index.TermPositions;
/*     */ import org.apache.lucene.store.Directory;
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ public class LuceneViewer
/*     */ {
/*     */   private String mIndexDir;
/*     */   private String mOutputFile;
/*     */   private TermFilters mTermFilters;
/*     */   private Console mConsole;
/*     */   private Set<Integer> mDocsIntersection;
/*     */   private IndexReader mIndexReader;
/*     */   private FileWriter mWriter;
/*     */   
/*     */   private static class TermFilters
/*     */   {
/*     */     private static class TermFilter
/*     */     {
/*     */       private String mField;
/*     */       private String mText;
/*     */       
/*     */       private TermFilter(String field, String text)
/*     */       {
/*  75 */         this.mField = field;
/*  76 */         this.mText = text;
/*     */       }
/*     */     }
/*     */     
/*  80 */     private List<TermFilter> mFilters = new ArrayList();
/*     */     
/*     */     private void addFilter(String field, String text) {
/*  83 */       this.mFilters.add(new TermFilter(field, text, null));
/*     */     }
/*     */     
/*     */     private List<TermFilter> getFilters() {
/*  87 */       return this.mFilters;
/*     */     }
/*     */   }
/*     */   
/*     */   public LuceneViewer(String indexName, String outfileName, TermFilters termFilters, Console console)
/*     */     throws Exception
/*     */   {
/*  94 */     this.mIndexDir = indexName;
/*  95 */     this.mOutputFile = outfileName;
/*  96 */     this.mTermFilters = termFilters;
/*  97 */     this.mConsole = console;
/*     */     
/*  99 */     this.mIndexReader = IndexReader.open(LuceneDirectory.open(new File(this.mIndexDir)));
/* 100 */     this.mWriter = new FileWriter(this.mOutputFile);
/*     */     
/* 102 */     if (hasFilters()) {
/* 103 */       this.mDocsIntersection = new HashSet();
/*     */     }
/*     */   }
/*     */   
/*     */   private List<LuceneViewer.TermFilters.TermFilter> getFilters() {
/* 108 */     List<LuceneViewer.TermFilters.TermFilter> filters = this.mTermFilters == null ? null : this.mTermFilters.getFilters();
/*     */     
/* 110 */     return filters;
/*     */   }
/*     */   
/*     */   private boolean hasFilters() {
/* 114 */     List<LuceneViewer.TermFilters.TermFilter> filters = this.mTermFilters == null ? null : this.mTermFilters.getFilters();
/*     */     
/* 116 */     if ((filters == null) || (filters.isEmpty())) {
/* 117 */       return false;
/*     */     }
/* 119 */     return true;
/*     */   }
/*     */   
/*     */   private boolean wantThisTerm(String termField, String termText)
/*     */   {
/* 124 */     if (!hasFilters()) {
/* 125 */       return true;
/*     */     }
/*     */     
/* 128 */     for (LuceneViewer.TermFilters.TermFilter termFilter : getFilters()) {
/* 129 */       String field = termFilter.mField;
/* 130 */       String text = termFilter.mText;
/* 131 */       boolean matched = ((field == null) || (field.equalsIgnoreCase(termField)) ? 1 : 0) & ((text == null) || (text.equalsIgnoreCase(termText)) ? 1 : 0);
/*     */       
/* 133 */       if (matched) {
/* 134 */         return true;
/*     */       }
/*     */     }
/*     */     
/* 138 */     return false;
/*     */   }
/*     */   
/*     */   private void closeIndexReader() throws IOException
/*     */   {
/* 143 */     this.mIndexReader.close();
/*     */   }
/*     */   
/*     */   private void outputBanner(String bannerText) throws IOException {
/* 147 */     outputLn();
/* 148 */     outputLn("==============================");
/* 149 */     outputLn(bannerText);
/* 150 */     outputLn("==============================");
/* 151 */     outputLn();
/*     */   }
/*     */   
/*     */   private void outputLn() throws IOException {
/* 155 */     output("\n");
/*     */   }
/*     */   
/*     */   private void outputLn(String out) throws IOException {
/* 159 */     output(out + "\n");
/*     */   }
/*     */   
/*     */   private void output(String out) throws IOException {
/* 163 */     this.mWriter.write(out);
/* 164 */     this.mWriter.flush();
/*     */   }
/*     */   
/*     */   private void closeOutputWriter() throws IOException {
/* 168 */     this.mWriter.close();
/*     */   }
/*     */   
/*     */   private void dump() throws Exception
/*     */   {
/* 173 */     outputLn("Index directory: " + this.mIndexDir);
/* 174 */     outputLn("Output file:     " + this.mOutputFile);
/* 175 */     dumpTermFilters();
/*     */     
/* 177 */     dumpFields();
/* 178 */     dumpDocuments();
/* 179 */     dumpTerms();
/*     */     
/* 181 */     dumpDocsIntersection();
/*     */     
/* 183 */     outputBanner("end");
/*     */     
/* 185 */     closeIndexReader();
/* 186 */     closeOutputWriter();
/*     */   }
/*     */   
/*     */   private void dumpTermFilters() throws IOException {
/* 190 */     if (!hasFilters()) {
/* 191 */       return;
/*     */     }
/*     */     
/* 194 */     outputLn("Term filters:");
/* 195 */     for (LuceneViewer.TermFilters.TermFilter termFilter : getFilters()) {
/* 196 */       String field = termFilter.mField;
/* 197 */       String text = termFilter.mText;
/* 198 */       outputLn("   (field: " + (field == null ? "" : field) + ") (text: " + (text == null ? "" : text) + ")");
/*     */     }
/*     */   }
/*     */   
/*     */   private void dumpFields() throws IOException
/*     */   {
/* 204 */     outputBanner("Fields");
/*     */     
/* 206 */     Collection<String> fieldNames = this.mIndexReader.getFieldNames(IndexReader.FieldOption.ALL);
/*     */     
/* 208 */     for (String fieldName : fieldNames) {
/* 209 */       outputLn("    " + fieldName.toString());
/*     */     }
/*     */   }
/*     */   
/*     */   private void dumpDocuments() throws IOException {
/* 214 */     outputBanner("Documents");
/*     */     
/* 216 */     int totalDocs = this.mIndexReader.numDocs();
/*     */     
/* 218 */     outputLn();
/* 219 */     outputLn("There are " + totalDocs + " documents in this index.");
/*     */     
/* 221 */     this.mConsole.debug("Total number of documents: " + totalDocs);
/* 222 */     for (int i = 0; i < totalDocs; i++) {
/* 223 */       Document doc = null;
/*     */       try {
/* 225 */         doc = this.mIndexReader.document(i, null);
/*     */       } catch (IllegalArgumentException e) {
/* 227 */         if ("attempt to access a deleted document".equals(e.getMessage())) {
/* 228 */           this.mConsole.warn("encountered exception while dumping document " + i + ": " + e.getMessage());
/*     */         }
/*     */         else {
/* 231 */           throw e;
/*     */         }
/*     */       }
/* 234 */       dumpDocument(i, doc);
/*     */       
/* 236 */       if ((i + 1) % 100 == 0) {
/* 237 */         this.mConsole.debug("Dumped " + (i + 1) + " documents");
/*     */       }
/*     */     }
/*     */   }
/*     */   
/*     */   private void dumpDocument(int docNum, Document doc) throws IOException
/*     */   {
/* 244 */     outputLn();
/* 245 */     outputLn("Document " + docNum);
/*     */     
/* 247 */     if (doc == null) {
/* 248 */       outputLn("    deleted");
/* 249 */       return;
/*     */     }
/*     */     
/*     */ 
/* 253 */     for (Fieldable field : doc.getFields()) {
/* 254 */       String fieldName = field.name();
/*     */       
/* 256 */       boolean isDate = "l.date".equals(fieldName);
/*     */       
/* 258 */       outputLn("    Field [" + fieldName + "]: " + field.toString());
/* 259 */       String[] values = doc.getValues(fieldName);
/* 260 */       if (values != null) {
/* 261 */         int i = 0;
/* 262 */         for (String value : values) {
/* 263 */           output("         (" + i++ + ") " + value);
/* 264 */           if (isDate) {
/*     */             try {
/* 266 */               Date date = DateTools.stringToDate(value);
/* 267 */               output(" (" + date.toString() + " (" + date.getTime() + "))");
/*     */             }
/*     */             catch (java.text.ParseException e) {
/* 270 */               if (!$assertionsDisabled) throw new AssertionError();
/*     */             }
/*     */           }
/* 273 */           outputLn();
/*     */         }
/*     */       }
/*     */     }
/*     */   }
/*     */   
/*     */ 
/*     */   private void computeDocsIntersection(Set<Integer> docs)
/*     */   {
/* 282 */     if (!hasFilters()) {
/* 283 */       return;
/*     */     }
/*     */     
/* 286 */     if (this.mDocsIntersection.isEmpty()) {
/* 287 */       this.mDocsIntersection = docs;
/*     */     } else {
/* 289 */       this.mDocsIntersection = SetUtil.intersect(this.mDocsIntersection, docs);
/*     */     }
/*     */   }
/*     */   
/*     */   private void dumpTerms() throws IOException {
/* 294 */     outputBanner("Terms (in Term.compareTo() order)");
/*     */     
/* 296 */     TermEnum terms = this.mIndexReader.terms();
/* 297 */     int order = 0;
/*     */     
/* 299 */     while (terms.next()) {
/* 300 */       order++;
/* 301 */       Term term = terms.term();
/* 302 */       String field = term.field();
/* 303 */       String text = term.text();
/*     */       
/* 305 */       if (wantThisTerm(field, text))
/*     */       {
/*     */ 
/*     */ 
/* 309 */         outputLn(order + " " + field + ": " + text);
/*     */         
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/* 328 */         outputLn("    document, frequency, <position>*");
/*     */         
/*     */ 
/* 331 */         Set<Integer> docNums = null;
/* 332 */         if (hasFilters()) {
/* 333 */           docNums = new HashSet();
/*     */         }
/*     */         
/* 336 */         TermPositions termPos = this.mIndexReader.termPositions(term);
/* 337 */         while (termPos.next()) {
/* 338 */           int docNum = termPos.doc();
/* 339 */           int freq = termPos.freq();
/*     */           
/* 341 */           if (docNums != null) {
/* 342 */             docNums.add(Integer.valueOf(docNum));
/*     */           }
/*     */           
/* 345 */           output("    " + docNum + ", " + freq + ", <");
/*     */           
/* 347 */           boolean first = true;
/* 348 */           for (int f = 0; f < freq; f++) {
/* 349 */             int positionInDoc = termPos.nextPosition();
/* 350 */             if (!first) {
/* 351 */               output(" ");
/*     */             } else {
/* 353 */               first = false;
/*     */             }
/* 355 */             output(positionInDoc + "");
/*     */           }
/* 357 */           outputLn(">");
/*     */         }
/* 359 */         termPos.close();
/*     */         
/* 361 */         if (docNums != null) {
/* 362 */           computeDocsIntersection(docNums);
/*     */         }
/*     */         
/* 365 */         outputLn();
/*     */         
/* 367 */         if (order % 1000 == 0) {
/* 368 */           this.mConsole.debug("Dumped " + order + " terms");
/*     */         }
/*     */       }
/*     */     }
/* 372 */     terms.close();
/*     */   }
/*     */   
/*     */   private void dumpDocsIntersection() throws IOException {
/* 376 */     if (this.mDocsIntersection == null) {
/* 377 */       return;
/*     */     }
/*     */     
/* 380 */     outputBanner("Documents in which all (filtered in) terms appear");
/*     */     
/* 382 */     List<Integer> sorted = new ArrayList(this.mDocsIntersection);
/* 383 */     Collections.sort(sorted);
/* 384 */     for (Integer docNum : sorted) {
/* 385 */       outputLn("    " + docNum);
/*     */     }
/*     */   }
/*     */   
/*     */ 
/*     */   private static class CLI
/*     */   {
/*     */     public static final int NUM_TERM_FILTERS = 10;
/*     */     
/*     */     public static final String O_ACTION = "a";
/*     */     public static final String O_HELP = "h";
/*     */     public static final String O_INPUT = "i";
/*     */     public static final String O_OUTPUT = "o";
/*     */     public static final String O_VERBOSE = "v";
/*     */     private static final String O_TERM_FILTER_FIELD_PREFIX = "f";
/*     */     private static final String O_TERM_FILTER_TEXT_PREFIX = "t";
/*     */     
/*     */     private Options getAllOptions()
/*     */     {
/* 404 */       return getOptions(true);
/*     */     }
/*     */     
/*     */     private Options getOptions(boolean includeHiddenOptions) {
/* 408 */       Options options = new Options();
/*     */       
/* 410 */       options.addOption("a", "action", true, "action, values are dump|check");
/* 411 */       options.addOption("h", "help", false, "input directory");
/* 412 */       options.addOption("i", "input", true, "input directory");
/* 413 */       options.addOption("o", "output", true, "output file");
/* 414 */       options.addOption("v", "verbose", false, "verbose mode");
/*     */       
/* 416 */       if (includeHiddenOptions) {
/* 417 */         for (Object option : getTermFilterOptions().getOptions()) {
/* 418 */           options.addOption((Option)option);
/*     */         }
/*     */       }
/*     */       
/* 422 */       return options;
/*     */     }
/*     */     
/*     */     private static String termFilterFieldOption(int i) {
/* 426 */       return "f" + i;
/*     */     }
/*     */     
/*     */     private static String termFilterTextOption(int i) {
/* 430 */       return "t" + i;
/*     */     }
/*     */     
/*     */     private Options getTermFilterOptions() {
/* 434 */       Options options = new Options();
/*     */       
/* 436 */       for (int i = 1; i <= 10; i++) {
/* 437 */         options.addOption(termFilterFieldOption(i), "field" + i, true, "field name of term filter " + i);
/*     */         
/* 439 */         options.addOption(termFilterTextOption(i), "text" + i, true, "text of term filter " + i);
/*     */       }
/*     */       
/*     */ 
/* 443 */       return options;
/*     */     }
/*     */     
/*     */     private boolean helpOptionSpecified(String[] args) {
/* 447 */       return (args != null) && (args.length == 1) && (("-h".equals(args[0])) || ("--help".equals(args[0])));
/*     */     }
/*     */     
/*     */     private void usage(boolean exit)
/*     */     {
/* 452 */       usage(null, exit);
/*     */     }
/*     */     
/*     */     protected String getCommandUsage() {
/* 456 */       return "zmjava com.zimbra.cs.index.LuceneViewer <options> [term filter options]";
/*     */     }
/*     */     
/*     */     private void usage(org.apache.commons.cli.ParseException e, boolean exit) {
/* 460 */       if (e != null) {
/* 461 */         System.err.println(e.getMessage());
/* 462 */         System.err.println();
/* 463 */         System.err.println();
/*     */       }
/*     */       
/* 466 */       PrintWriter pw = new PrintWriter(System.err, true);
/* 467 */       HelpFormatter formatter = new HelpFormatter();
/* 468 */       formatter.setWidth(80);
/* 469 */       formatter.printHelp(pw, formatter.getWidth(), getCommandUsage(), null, getOptions(false), formatter.getLeftPadding(), formatter.getDescPadding(), null);
/*     */       
/* 471 */       pw.flush();
/*     */       
/* 473 */       System.err.println();
/* 474 */       System.err.println("term filter: ");
/* 475 */       System.err.println("    - each f[n], t[n] pair represents a term filter, f[n], t[n] don't have to both exist");
/* 476 */       System.err.println("    - f[n] represents a term field name, t[n] represents a term text");
/* 477 */       System.err.println("    - the final filter is formed by ORing all term filters together");
/* 478 */       System.err.println("    - maximum of 10 term filters are allowed");
/*     */       
/* 480 */       System.err.println();
/* 481 */       System.err.println("    examples:");
/* 482 */       System.err.println("        -f1 l.content -t1 foo (term l.content=foo)");
/* 483 */       System.err.println("        -f2 from              (all terms with from as the field name)");
/* 484 */       System.err.println("        -t3 bar               (all terms with bar as the text)");
/*     */       
/*     */ 
/* 487 */       System.err.println();
/* 488 */       System.err.println();
/* 489 */       System.err.println("Sample command lines:");
/* 490 */       System.err.println("zmjava com.zimbra.cs.index.LuceneViewer -a dump -i /opt/zimbra/index/0/2/index/0 -o /tmp/user1-index-dump.txt");
/* 491 */       System.err.println("zmjava com.zimbra.cs.index.LuceneViewer -a dump -v -f1 l.content -t1 jay -f2 subject -t2 howdy -i /opt/zimbra/index/0/2/index/0 -o /tmp/user1-index-dump.txt");
/* 492 */       System.err.println("zmjava com.zimbra.cs.index.LuceneViewer -a dump -f1 from jay@test.com -i /opt/zimbra/index/0/2/index/0 -o /tmp/user1-index-dump.txt");
/*     */       
/* 494 */       if (exit) {
/* 495 */         System.exit(1);
/*     */       }
/*     */     }
/*     */     
/*     */     protected CommandLine getCommandLine(String[] args) {
/* 500 */       CommandLineParser clParser = new GnuParser();
/* 501 */       CommandLine cl = null;
/*     */       
/* 503 */       Options opts = getAllOptions();
/*     */       try {
/* 505 */         cl = clParser.parse(opts, args);
/*     */       } catch (org.apache.commons.cli.ParseException e) {
/* 507 */         if (helpOptionSpecified(args)) {
/* 508 */           usage(true);
/*     */         } else {
/* 510 */           usage(e, true);
/*     */         }
/*     */       }
/*     */       
/* 514 */       return cl;
/*     */     }
/*     */     
/*     */     static LuceneViewer.TermFilters getTermFilters(CommandLine cl) {
/* 518 */       LuceneViewer.TermFilters termFilters = new LuceneViewer.TermFilters(null);
/*     */       
/* 520 */       for (int i = 1; i <= 10; i++) {
/* 521 */         String fieldOption = termFilterFieldOption(i);
/* 522 */         String textOption = termFilterTextOption(i);
/* 523 */         if ((cl.hasOption(fieldOption)) || (cl.hasOption(textOption))) {
/* 524 */           LuceneViewer.TermFilters.access$700(termFilters, cl.hasOption(fieldOption) ? cl.getOptionValue(fieldOption) : null, cl.hasOption(textOption) ? cl.getOptionValue(textOption) : null);
/*     */         }
/*     */       }
/*     */       
/*     */ 
/* 529 */       return termFilters;
/*     */     }
/*     */   }
/*     */   
/*     */   private static class Console {
/*     */     private boolean mVerbose;
/*     */     
/*     */     Console(boolean verbose) {
/* 537 */       this.mVerbose = verbose;
/*     */     }
/*     */     
/*     */     private void debug(String text) {
/* 541 */       if (this.mVerbose) {
/* 542 */         System.out.println(text);
/*     */       }
/*     */     }
/*     */     
/*     */     private void info(String text) {
/* 547 */       System.out.println(text);
/*     */     }
/*     */     
/*     */     private void warn(String text) {
/* 551 */       System.out.println(text);
/*     */     }
/*     */   }
/*     */   
/*     */   private static void doCheck(CommandLine cl) throws Exception
/*     */   {
/* 557 */     Console console = new Console(cl.hasOption("v"));
/*     */     
/* 559 */     String indexDir = cl.getOptionValue("i");
/* 560 */     console.info("Checking index " + indexDir);
/*     */     
/* 562 */     Directory dir = null;
/*     */     try {
/* 564 */       dir = LuceneDirectory.open(new File(indexDir));
/*     */     } catch (Throwable t) {
/* 566 */       console.info("ERROR: could not open directory \"" + indexDir + "\"; exiting");
/*     */       
/* 568 */       t.printStackTrace(System.out);
/* 569 */       System.exit(1);
/*     */     }
/*     */     
/* 572 */     CheckIndex checker = new CheckIndex(dir);
/* 573 */     checker.setInfoStream(System.out);
/*     */     
/* 575 */     CheckIndex.Status result = checker.checkIndex();
/* 576 */     console.info("Result:" + (result.clean ? "clean" : "not clean"));
/*     */   }
/*     */   
/*     */   private static void doDump(CommandLine cl) throws Exception {
/* 580 */     Console console = new Console(cl.hasOption("v"));
/*     */     
/* 582 */     String indexDir = cl.getOptionValue("i");
/* 583 */     String outputFile = cl.getOptionValue("o");
/*     */     
/* 585 */     TermFilters termFilters = CLI.getTermFilters(cl);
/*     */     
/* 587 */     console.info("Dumping index directory: " + indexDir);
/* 588 */     console.info("Output file: " + outputFile);
/*     */     
/* 590 */     LuceneViewer viewer = new LuceneViewer(indexDir, outputFile, termFilters, console);
/*     */     
/* 592 */     viewer.dump();
/*     */     
/* 594 */     console.info("all done");
/*     */   }
/*     */   
/*     */   public static void main(String[] args) throws Exception
/*     */   {
/* 599 */     CLI cli = new CLI(null);
/* 600 */     CommandLine cl = cli.getCommandLine(args);
/*     */     
/* 602 */     if (!cl.hasOption("a")) {
/* 603 */       cli.usage(new org.apache.commons.cli.ParseException("missing required option a"), true);
/*     */     }
/*     */     
/*     */ 
/* 607 */     String action = cl.getOptionValue("a");
/* 608 */     if ("dump".equals(action)) {
/* 609 */       doDump(cl);
/* 610 */     } else if ("check".equals(action)) {
/* 611 */       doCheck(cl);
/*     */     } else {
/* 613 */       cli.usage(new org.apache.commons.cli.ParseException("invalid option " + action), true);
/*     */     }
/*     */   }
/*     */ }


/* Location:              /home/mint/zimbrastore.jar!/com/zimbra/cs/index/LuceneViewer.class
 * Java compiler version: 7 (51.0)
 * JD-Core Version:       0.7.1
 */